#!/usr/bin/env python
import spade, urllib2, time, sys, sqlite3, re, json
from bs4 import BeautifulSoup, Comment
from collections import deque


class DataAgent(spade.Agent.Agent):

    class directoryServices(spade.Behaviour.OneShotBehaviour):
        def _process(self):
            dad = spade.DF.DfAgentDescription()
            sd = spade.DF.ServiceDescription()
            sd.setName("location")
            sd.setType("provider")
            dad.addService(sd)
            sd = spade.DF.ServiceDescription()
            sd.setName("hotelurl")
            sd.setType("provider")
            dad.addService(sd)
            sd = spade.DF.ServiceDescription()
            sd.setName("reviewurl")
            sd.setType("provider")
            dad.addService(sd)
            dad.setAID(self.myAgent.getAID())
            res = self.myAgent.registerService(dad)
            print "Started location crawler"

    class provideLocation(spade.Behaviour.EventBehaviour):
        def _process(self):
            msg = self._receive(True)
            rep = msg.createReply()
            if len(self.myAgent._locations_) > 0:
                rep.setContent(self.myAgent._locations_.pop())
            else:
                rep.setContent("")
                rep.setPerformative("refuse")
            self.myAgent.send(rep)

    class provideHotelURL(spade.Behaviour.EventBehaviour):
        def _process(self):
            msg = self._receive(True)
            rep = msg.createReply()
            if len(self.myAgent._known_hotels_) > 0:
                hotelurl = self.myAgent._known_hotels_.pop()
                self.myAgent._hotel_buffer_.append(hotelurl)
                rep.setContent(hotelurl)
            else:
                rep.setContent("")
                rep.setPerformative("refuse")
            self.myAgent.send(rep)

    class provideReviewURL(spade.Behaviour.EventBehaviour):
        def _process(self):
            msg = self._receive(True)
            rep = msg.createReply()
            if len(self.myAgent._known_reviews_) > 0:
                reviewurl = self.myAgent._known_reviews_.pop()
                self.myAgent._review_buffer_.append(reviewurl)
                rep.setContent(reviewurl)
            else:
                rep.setContent("")
                rep.setPerformative("refuse")
            self.myAgent.send(rep)

    class receiveHotelURL(spade.Behaviour.EventBehaviour):
        def _process(self):
            msg = self._receive(True)
            #links = [link.strip(" u'") for link in msg.getContent().strip('[]').split(',')]
            links = json.loads(msg.getContent())
            for link in links:
                self.myAgent._known_hotels_.append(link)
                print "Added hotel: " + link

    class receiveReviewURL(spade.Behaviour.EventBehaviour):
        def _process(self):
            msg = self._receive(True)
            hotelurl = msg.getProtocol()
            if hotelurl in self.myAgent._hotel_buffer_:
                self.myAgent._visited_hotels_.append(self.myAgent._hotel_buffer_.pop(self.myAgent._hotel_buffer_.index(hotelurl)))
            else: print "Got stray message"
            #links = [link.strip(" u'") for link in msg.getContent().strip('[]').split(',')]
            links = json.loads(msg.getContent())
            for link in links:
                self.myAgent._known_reviews_.append(link)
                print "Added review: " + link

    class receiveReviewInformation(spade.Behaviour.EventBehaviour):
        def _process(self):
            msg = self._receive(True)
            reviewurl = msg.getProtocol()
            if reviewurl in self.myAgent._review_buffer_:
                self.myAgent._visited_reviews_.append(self.myAgent._review_buffer_.pop(self.myAgent._review_buffer_.index(reviewurl)))
            else:
                print "Got stray message: " #+ reviewurl
                #print self.myAgent._review_buffer_
            #message = msg.getContent().split("&#%")
            message = json.loads(msg.getContent())
            if len(message) > 18:
                Data = (message[0], message[1], float(message[2]), float(message[3]),
                    float(message[4]), float(message[5]), float(message[6]),
                    float(message[7]), float(message[8]), float(message[9]),
                    message[10], message[11], float(message[12]),
                    float(message[13]), float(message[14]), float(message[15]),
                    float(message[16]), float(message[17]), float(message[18]),)
                print "Got review " + message[0] + " correctly"
                self.myAgent._information_.append(Data)
            else: print "Got failure"

    class addInformationToDatabase(spade.Behaviour.PeriodicBehaviour):
        def onStart(self):
            self.conn = sqlite3.connect('Hotels_g189413_Crete.db')
            self.c = self.conn.cursor()
            try:
                self.c.execute('''CREATE TABLE reviews (Review_ID text, Hotel_ID text, Hotel_Star real, Hotel_Rating real,
                Hotel_Clean real, Hotel_Service real, Hotel_Location real, Hotel_Rooms real, Hotel_Sleep real, Hotel_Value real,
                Review_Quote text, Review_Text text, Review_Rating real, Review_Clean real, Review_Service real, Review_Location real,
                Review_Rooms real, Review_Sleep real, Review_Value real)''')
            except:
                print "Table exists"
        def _onTick(self):
            try:
                while len(self.myAgent._information_) > 0:
                    data = self.myAgent._information_.pop()
                    self.c.execute("INSERT INTO Reviews VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)", data)
                    print "Added information"
                    self.conn.commit()
            except:
                print "database exception"
        def onEnd(self):
            self.conn.close()

    class printStatus(spade.Behaviour.PeriodicBehaviour):
        def _onTick(self):
            print "------------Status-------------"
            print "Hotels in buffer: " + str(len(self.myAgent._known_hotels_))
            print "Reviews in buffer: " + str(len(self.myAgent._known_reviews_))
            print "-------------------------------"
            with open('visited_hotels.txt', 'w') as vh:
                for link in self.myAgent._visited_hotels_:
                    vh.write(link.strip()+"\n")
            with open('known_hotels.txt', 'w') as kh:
                for link in self.myAgent._known_hotels_:
                    kh.write(link.strip()+"\n")
            with open('buffer_hotels.txt', 'w') as hb:
                for link in self.myAgent._hotel_buffer_:
                    hb.write(link.strip()+"\n")
            with open('visited_reviews.txt', 'w') as vr:
                for link in self.myAgent._visited_reviews_:
                    vr.write(link.strip()+"\n")
            with open('known_reviews.txt', 'w') as kr:
                for link in self.myAgent._known_reviews_:
                    kr.write(link.strip()+"\n")
            with open('buffer_reviews.txt', 'w') as rb:
                for link in self.myAgent._review_buffer_:
                    rb.write(link.strip()+"\n")

    class getstraymessages(spade.Behaviour.Behaviour):
        def _process(self):
            msg = self._receive(True, 10)
            print "stray message" + msg.getContent()

    def _setup(self):
        self._visited_hotels_ = []
        self._known_hotels_ = []
        self._hotel_buffer_ = []
        self._known_reviews_ = []
        self._review_buffer_ = []
        self._visited_reviews_ = []
        self._locations_ = ["/Hotels-g189413-Crete-Hotels.html"]
        self._information_ = []
        with open('visited_hotels.txt', 'r') as vh:
            for link in vh:
                self._visited_hotels_.append(link.strip("\n"))
        with open('known_hotels.txt', 'r') as kh:
            for link in kh:
                self._known_hotels_.append(link.strip("\n"))
        with open('buffer_hotels.txt', 'r') as hb:
            for link in hb:
                self._known_hotels_.append(link.strip("\n"))
        with open('visited_reviews.txt', 'r') as vr:
            for link in vr:
                self._visited_reviews_.append(link.strip("\n"))
        with open('known_reviews.txt', 'r') as kr:
            for link in kr:
                self._known_reviews_.append(link.strip("\n"))
        with open('buffer_reviews.txt', 'r') as rb:
            for link in rb:
                self._known_reviews_.append(link.strip("\n"))
        provide_location_template = spade.Behaviour.ACLTemplate()
        provide_location_template.setOntology("location")
        provide_location_template.setPerformative("request")
        provide_hotelurl_template = spade.Behaviour.ACLTemplate()
        provide_hotelurl_template.setOntology("hotel")
        provide_hotelurl_template.setPerformative("request")
        provide_reviewurl_template = spade.Behaviour.ACLTemplate()
        provide_reviewurl_template.setOntology("review")
        provide_reviewurl_template.setPerformative("request")
        receive_information_template = spade.Behaviour.ACLTemplate()
        receive_information_template.setOntology("review")
        receive_information_template.setPerformative("inform")
        receive_hotelurl_template = spade.Behaviour.ACLTemplate()
        receive_hotelurl_template.setOntology("location")
        receive_hotelurl_template.setPerformative("inform")
        receive_reviewurl_template = spade.Behaviour.ACLTemplate()
        receive_reviewurl_template.setOntology("hotel")
        receive_reviewurl_template.setPerformative("inform")
        self.addBehaviour(self.directoryServices(),None)
        self.addBehaviour(self.addInformationToDatabase(10),None)
        self.addBehaviour(self.printStatus(10),None)
        self.addBehaviour(self.provideLocation(),spade.Behaviour.MessageTemplate(provide_location_template))
        self.addBehaviour(self.provideHotelURL(),spade.Behaviour.MessageTemplate(provide_hotelurl_template))
        self.addBehaviour(self.provideReviewURL(),spade.Behaviour.MessageTemplate(provide_reviewurl_template))
        self.addBehaviour(self.receiveHotelURL(),spade.Behaviour.MessageTemplate(receive_hotelurl_template))
        self.addBehaviour(self.receiveReviewURL(),spade.Behaviour.MessageTemplate(receive_reviewurl_template))
        self.addBehaviour(self.receiveReviewInformation(),spade.Behaviour.MessageTemplate(receive_information_template))
        self.setDefaultBehaviour(self.getstraymessages())

if __name__ == "__main__":
    a = DataAgent("coop@127.0.0.1", "secret")
    a.start()
    Alive = True
    while Alive:
        try: time.sleep(1)
        except KeyboardInterrupt: Alive = False
    a.stop()
    sys.exit(0)
